{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json\n",
    "\n",
    "offical_names = ['MinerU-0.9.3', 'Marker-0.2.17', 'Marker-1.2.3', 'Mathpix', 'docling', 'GOT-OCR', 'Nougat', 'GPT4o', 'Qwen2-VL-72B', 'InternVL2-Llama3-76B']\n",
    "ocr_types = ['mineru1110_docyolo', 'marker', 'marker0106', 'mathpix1108', 'docling', 'GOT', 'nougat', 'gpt4o_long_prompt', 'QwenVL_long_prompt', 'internvl_long_prompt']\n",
    "# offical_names = ocr_types\n",
    "\n",
    "result_folder = './result'\n",
    "\n",
    "# match_name = 'no_split'\n",
    "match_name = 'quick_match'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_block_Edit_dist_EN</th>\n",
       "      <th>text_block_Edit_dist_CH</th>\n",
       "      <th>display_formula_Edit_dist_EN</th>\n",
       "      <th>display_formula_Edit_dist_CH</th>\n",
       "      <th>display_formula_CDM_EN</th>\n",
       "      <th>display_formula_CDM_CH</th>\n",
       "      <th>table_TEDS_EN</th>\n",
       "      <th>table_TEDS_CH</th>\n",
       "      <th>table_Edit_dist_EN</th>\n",
       "      <th>table_Edit_dist_CH</th>\n",
       "      <th>reading_order_Edit_dist_EN</th>\n",
       "      <th>reading_order_Edit_dist_CH</th>\n",
       "      <th>overall_EN</th>\n",
       "      <th>overall_CH</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>MinerU-0.9.3</th>\n",
       "      <td>0.061</td>\n",
       "      <td>0.211</td>\n",
       "      <td>0.278</td>\n",
       "      <td>0.577</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>78.622</td>\n",
       "      <td>62.145</td>\n",
       "      <td>0.180</td>\n",
       "      <td>0.344</td>\n",
       "      <td>0.079</td>\n",
       "      <td>0.288</td>\n",
       "      <td>0.150</td>\n",
       "      <td>0.355</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Marker-0.2.17</th>\n",
       "      <td>0.141</td>\n",
       "      <td>0.303</td>\n",
       "      <td>0.667</td>\n",
       "      <td>0.868</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>53.470</td>\n",
       "      <td>44.892</td>\n",
       "      <td>0.678</td>\n",
       "      <td>0.714</td>\n",
       "      <td>0.138</td>\n",
       "      <td>0.306</td>\n",
       "      <td>0.406</td>\n",
       "      <td>0.548</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Marker-1.2.3</th>\n",
       "      <td>0.080</td>\n",
       "      <td>0.315</td>\n",
       "      <td>0.530</td>\n",
       "      <td>0.883</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>67.564</td>\n",
       "      <td>49.207</td>\n",
       "      <td>0.619</td>\n",
       "      <td>0.685</td>\n",
       "      <td>0.114</td>\n",
       "      <td>0.340</td>\n",
       "      <td>0.336</td>\n",
       "      <td>0.556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mathpix</th>\n",
       "      <td>0.101</td>\n",
       "      <td>0.358</td>\n",
       "      <td>0.306</td>\n",
       "      <td>0.454</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>77.048</td>\n",
       "      <td>67.148</td>\n",
       "      <td>0.243</td>\n",
       "      <td>0.320</td>\n",
       "      <td>0.105</td>\n",
       "      <td>0.275</td>\n",
       "      <td>0.189</td>\n",
       "      <td>0.352</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>docling</th>\n",
       "      <td>0.416</td>\n",
       "      <td>0.987</td>\n",
       "      <td>0.999</td>\n",
       "      <td>1.000</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>61.277</td>\n",
       "      <td>24.972</td>\n",
       "      <td>0.627</td>\n",
       "      <td>0.810</td>\n",
       "      <td>0.313</td>\n",
       "      <td>0.837</td>\n",
       "      <td>0.589</td>\n",
       "      <td>0.909</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GOT-OCR</th>\n",
       "      <td>0.191</td>\n",
       "      <td>0.315</td>\n",
       "      <td>0.360</td>\n",
       "      <td>0.528</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>53.193</td>\n",
       "      <td>47.201</td>\n",
       "      <td>0.459</td>\n",
       "      <td>0.520</td>\n",
       "      <td>0.143</td>\n",
       "      <td>0.280</td>\n",
       "      <td>0.289</td>\n",
       "      <td>0.411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Nougat</th>\n",
       "      <td>0.367</td>\n",
       "      <td>0.998</td>\n",
       "      <td>0.488</td>\n",
       "      <td>0.941</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>39.902</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.572</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.384</td>\n",
       "      <td>0.954</td>\n",
       "      <td>0.453</td>\n",
       "      <td>0.973</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GPT4o</th>\n",
       "      <td>0.146</td>\n",
       "      <td>0.409</td>\n",
       "      <td>0.425</td>\n",
       "      <td>0.606</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>71.977</td>\n",
       "      <td>62.870</td>\n",
       "      <td>0.234</td>\n",
       "      <td>0.329</td>\n",
       "      <td>0.128</td>\n",
       "      <td>0.251</td>\n",
       "      <td>0.233</td>\n",
       "      <td>0.399</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Qwen2-VL-72B</th>\n",
       "      <td>0.253</td>\n",
       "      <td>0.251</td>\n",
       "      <td>0.468</td>\n",
       "      <td>0.572</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>59.503</td>\n",
       "      <td>66.413</td>\n",
       "      <td>0.551</td>\n",
       "      <td>0.518</td>\n",
       "      <td>0.254</td>\n",
       "      <td>0.223</td>\n",
       "      <td>0.381</td>\n",
       "      <td>0.391</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>InternVL2-Llama3-76B</th>\n",
       "      <td>0.353</td>\n",
       "      <td>0.290</td>\n",
       "      <td>0.543</td>\n",
       "      <td>0.701</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>63.004</td>\n",
       "      <td>60.243</td>\n",
       "      <td>0.547</td>\n",
       "      <td>0.555</td>\n",
       "      <td>0.317</td>\n",
       "      <td>0.228</td>\n",
       "      <td>0.440</td>\n",
       "      <td>0.443</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      text_block_Edit_dist_EN  text_block_Edit_dist_CH  \\\n",
       "MinerU-0.9.3                            0.061                    0.211   \n",
       "Marker-0.2.17                           0.141                    0.303   \n",
       "Marker-1.2.3                            0.080                    0.315   \n",
       "Mathpix                                 0.101                    0.358   \n",
       "docling                                 0.416                    0.987   \n",
       "GOT-OCR                                 0.191                    0.315   \n",
       "Nougat                                  0.367                    0.998   \n",
       "GPT4o                                   0.146                    0.409   \n",
       "Qwen2-VL-72B                            0.253                    0.251   \n",
       "InternVL2-Llama3-76B                    0.353                    0.290   \n",
       "\n",
       "                      display_formula_Edit_dist_EN  \\\n",
       "MinerU-0.9.3                                 0.278   \n",
       "Marker-0.2.17                                0.667   \n",
       "Marker-1.2.3                                 0.530   \n",
       "Mathpix                                      0.306   \n",
       "docling                                      0.999   \n",
       "GOT-OCR                                      0.360   \n",
       "Nougat                                       0.488   \n",
       "GPT4o                                        0.425   \n",
       "Qwen2-VL-72B                                 0.468   \n",
       "InternVL2-Llama3-76B                         0.543   \n",
       "\n",
       "                      display_formula_Edit_dist_CH display_formula_CDM_EN  \\\n",
       "MinerU-0.9.3                                 0.577                      -   \n",
       "Marker-0.2.17                                0.868                      -   \n",
       "Marker-1.2.3                                 0.883                      -   \n",
       "Mathpix                                      0.454                      -   \n",
       "docling                                      1.000                      -   \n",
       "GOT-OCR                                      0.528                      -   \n",
       "Nougat                                       0.941                      -   \n",
       "GPT4o                                        0.606                      -   \n",
       "Qwen2-VL-72B                                 0.572                      -   \n",
       "InternVL2-Llama3-76B                         0.701                      -   \n",
       "\n",
       "                     display_formula_CDM_CH  table_TEDS_EN  table_TEDS_CH  \\\n",
       "MinerU-0.9.3                              -         78.622         62.145   \n",
       "Marker-0.2.17                             -         53.470         44.892   \n",
       "Marker-1.2.3                              -         67.564         49.207   \n",
       "Mathpix                                   -         77.048         67.148   \n",
       "docling                                   -         61.277         24.972   \n",
       "GOT-OCR                                   -         53.193         47.201   \n",
       "Nougat                                    -         39.902          0.000   \n",
       "GPT4o                                     -         71.977         62.870   \n",
       "Qwen2-VL-72B                              -         59.503         66.413   \n",
       "InternVL2-Llama3-76B                      -         63.004         60.243   \n",
       "\n",
       "                      table_Edit_dist_EN  table_Edit_dist_CH  \\\n",
       "MinerU-0.9.3                       0.180               0.344   \n",
       "Marker-0.2.17                      0.678               0.714   \n",
       "Marker-1.2.3                       0.619               0.685   \n",
       "Mathpix                            0.243               0.320   \n",
       "docling                            0.627               0.810   \n",
       "GOT-OCR                            0.459               0.520   \n",
       "Nougat                             0.572               1.000   \n",
       "GPT4o                              0.234               0.329   \n",
       "Qwen2-VL-72B                       0.551               0.518   \n",
       "InternVL2-Llama3-76B               0.547               0.555   \n",
       "\n",
       "                      reading_order_Edit_dist_EN  reading_order_Edit_dist_CH  \\\n",
       "MinerU-0.9.3                               0.079                       0.288   \n",
       "Marker-0.2.17                              0.138                       0.306   \n",
       "Marker-1.2.3                               0.114                       0.340   \n",
       "Mathpix                                    0.105                       0.275   \n",
       "docling                                    0.313                       0.837   \n",
       "GOT-OCR                                    0.143                       0.280   \n",
       "Nougat                                     0.384                       0.954   \n",
       "GPT4o                                      0.128                       0.251   \n",
       "Qwen2-VL-72B                               0.254                       0.223   \n",
       "InternVL2-Llama3-76B                       0.317                       0.228   \n",
       "\n",
       "                      overall_EN  overall_CH  \n",
       "MinerU-0.9.3               0.150       0.355  \n",
       "Marker-0.2.17              0.406       0.548  \n",
       "Marker-1.2.3               0.336       0.556  \n",
       "Mathpix                    0.189       0.352  \n",
       "docling                    0.589       0.909  \n",
       "GOT-OCR                    0.289       0.411  \n",
       "Nougat                     0.453       0.973  \n",
       "GPT4o                      0.233       0.399  \n",
       "Qwen2-VL-72B               0.381       0.391  \n",
       "InternVL2-Llama3-76B       0.440       0.443  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# overall result\n",
    "\n",
    "dict_list = []\n",
    "\n",
    "for ocr_type in ocr_types:\n",
    "    result_path = os.path.join(result_folder, f'{ocr_type}_{match_name}_metric_result.json')\n",
    "    \n",
    "    with open(result_path, 'r') as f:\n",
    "        result = json.load(f)\n",
    "    \n",
    "    save_dict = {}\n",
    "    en_overall = []\n",
    "    ch_overall = []\n",
    "    for category_type, metric in [(\"text_block\", \"Edit_dist\"), (\"display_formula\", \"Edit_dist\"), (\"display_formula\", \"CDM\"), (\"table\", \"TEDS\"), (\"table\", \"Edit_dist\"), (\"reading_order\", \"Edit_dist\")]:\n",
    "        if metric == 'CDM':\n",
    "            save_dict[category_type+'_'+metric+'_EN'] = '-'\n",
    "            save_dict[category_type+'_'+metric+'_CH'] = '-'\n",
    "        elif metric == \"TEDS\":\n",
    "            save_dict[category_type+'_'+metric+'_EN'] = result[category_type][\"page\"][metric][\"language: english\"] * 100\n",
    "            save_dict[category_type+'_'+metric+'_CH'] = result[category_type][\"page\"][metric][\"language: simplified_chinese\"] * 100\n",
    "            # save_dict[category_type+'_'+metric+'_EN'] = '-'\n",
    "            # save_dict[category_type+'_'+metric+'_CH'] = '-'\n",
    "        else:\n",
    "            save_dict[category_type+'_'+metric+'_EN'] = result[category_type][\"page\"][metric].get(\"language: english\", np.NaN)\n",
    "            save_dict[category_type+'_'+metric+'_CH'] = result[category_type][\"page\"][metric].get(\"language: simplified_chinese\",np.NaN)\n",
    "        if metric == \"Edit_dist\":\n",
    "            en_overall.append(result[category_type][\"page\"][metric].get(\"language: english\", np.NaN))\n",
    "            ch_overall.append(result[category_type][\"page\"][metric].get(\"language: simplified_chinese\",np.NaN))\n",
    "    \n",
    "    save_dict['overall_EN'] = sum(en_overall) / len(en_overall)\n",
    "    save_dict['overall_CH'] = sum(ch_overall) / len(ch_overall)\n",
    "    dict_list.append(save_dict)\n",
    "    \n",
    "df = pd.DataFrame(dict_list, index=offical_names).round(3)\n",
    "# df.to_csv('./overall.csv')\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/mnt/petrelfs/ouyanglinke/.local/lib/python3.7/site-packages/ipykernel_launcher.py:18: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data_source: book</th>\n",
       "      <th>data_source: PPT2PDF</th>\n",
       "      <th>data_source: research_report</th>\n",
       "      <th>data_source: colorful_textbook</th>\n",
       "      <th>data_source: exam_paper</th>\n",
       "      <th>data_source: magazine</th>\n",
       "      <th>data_source: academic_literature</th>\n",
       "      <th>data_source: note</th>\n",
       "      <th>data_source: newspaper</th>\n",
       "      <th>mean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>MinerU-0.9.3</th>\n",
       "      <td>0.055</td>\n",
       "      <td>0.124</td>\n",
       "      <td>0.033</td>\n",
       "      <td>0.102</td>\n",
       "      <td>0.159</td>\n",
       "      <td>0.072</td>\n",
       "      <td>0.025</td>\n",
       "      <td>0.984</td>\n",
       "      <td>0.148</td>\n",
       "      <td>0.189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Marker-0.2.17</th>\n",
       "      <td>0.188</td>\n",
       "      <td>0.327</td>\n",
       "      <td>0.087</td>\n",
       "      <td>0.292</td>\n",
       "      <td>0.423</td>\n",
       "      <td>0.134</td>\n",
       "      <td>0.102</td>\n",
       "      <td>0.470</td>\n",
       "      <td>0.270</td>\n",
       "      <td>0.255</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Marker-1.2.3</th>\n",
       "      <td>0.074</td>\n",
       "      <td>0.340</td>\n",
       "      <td>0.089</td>\n",
       "      <td>0.319</td>\n",
       "      <td>0.452</td>\n",
       "      <td>0.153</td>\n",
       "      <td>0.059</td>\n",
       "      <td>0.651</td>\n",
       "      <td>0.192</td>\n",
       "      <td>0.259</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mathpix</th>\n",
       "      <td>0.131</td>\n",
       "      <td>0.168</td>\n",
       "      <td>0.202</td>\n",
       "      <td>0.199</td>\n",
       "      <td>0.278</td>\n",
       "      <td>0.138</td>\n",
       "      <td>0.091</td>\n",
       "      <td>0.631</td>\n",
       "      <td>0.648</td>\n",
       "      <td>0.276</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>docling</th>\n",
       "      <td>0.709</td>\n",
       "      <td>0.916</td>\n",
       "      <td>0.994</td>\n",
       "      <td>0.801</td>\n",
       "      <td>0.853</td>\n",
       "      <td>0.720</td>\n",
       "      <td>0.326</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.996</td>\n",
       "      <td>0.813</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GOT-OCR</th>\n",
       "      <td>0.117</td>\n",
       "      <td>0.222</td>\n",
       "      <td>0.067</td>\n",
       "      <td>0.132</td>\n",
       "      <td>0.204</td>\n",
       "      <td>0.198</td>\n",
       "      <td>0.179</td>\n",
       "      <td>0.388</td>\n",
       "      <td>0.771</td>\n",
       "      <td>0.253</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Nougat</th>\n",
       "      <td>0.739</td>\n",
       "      <td>0.958</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.820</td>\n",
       "      <td>0.930</td>\n",
       "      <td>0.830</td>\n",
       "      <td>0.214</td>\n",
       "      <td>0.991</td>\n",
       "      <td>0.871</td>\n",
       "      <td>0.817</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GPT4o</th>\n",
       "      <td>0.163</td>\n",
       "      <td>0.163</td>\n",
       "      <td>0.348</td>\n",
       "      <td>0.187</td>\n",
       "      <td>0.281</td>\n",
       "      <td>0.173</td>\n",
       "      <td>0.146</td>\n",
       "      <td>0.607</td>\n",
       "      <td>0.751</td>\n",
       "      <td>0.313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Qwen2-VL-72B</th>\n",
       "      <td>0.096</td>\n",
       "      <td>0.080</td>\n",
       "      <td>0.145</td>\n",
       "      <td>0.148</td>\n",
       "      <td>0.219</td>\n",
       "      <td>0.065</td>\n",
       "      <td>0.315</td>\n",
       "      <td>0.298</td>\n",
       "      <td>0.790</td>\n",
       "      <td>0.240</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>InternVL2-Llama3-76B</th>\n",
       "      <td>0.216</td>\n",
       "      <td>0.098</td>\n",
       "      <td>0.162</td>\n",
       "      <td>0.184</td>\n",
       "      <td>0.247</td>\n",
       "      <td>0.150</td>\n",
       "      <td>0.419</td>\n",
       "      <td>0.226</td>\n",
       "      <td>0.903</td>\n",
       "      <td>0.289</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      data_source: book  data_source: PPT2PDF  \\\n",
       "MinerU-0.9.3                      0.055                 0.124   \n",
       "Marker-0.2.17                     0.188                 0.327   \n",
       "Marker-1.2.3                      0.074                 0.340   \n",
       "Mathpix                           0.131                 0.168   \n",
       "docling                           0.709                 0.916   \n",
       "GOT-OCR                           0.117                 0.222   \n",
       "Nougat                            0.739                 0.958   \n",
       "GPT4o                             0.163                 0.163   \n",
       "Qwen2-VL-72B                      0.096                 0.080   \n",
       "InternVL2-Llama3-76B              0.216                 0.098   \n",
       "\n",
       "                      data_source: research_report  \\\n",
       "MinerU-0.9.3                                 0.033   \n",
       "Marker-0.2.17                                0.087   \n",
       "Marker-1.2.3                                 0.089   \n",
       "Mathpix                                      0.202   \n",
       "docling                                      0.994   \n",
       "GOT-OCR                                      0.067   \n",
       "Nougat                                       1.000   \n",
       "GPT4o                                        0.348   \n",
       "Qwen2-VL-72B                                 0.145   \n",
       "InternVL2-Llama3-76B                         0.162   \n",
       "\n",
       "                      data_source: colorful_textbook  data_source: exam_paper  \\\n",
       "MinerU-0.9.3                                   0.102                    0.159   \n",
       "Marker-0.2.17                                  0.292                    0.423   \n",
       "Marker-1.2.3                                   0.319                    0.452   \n",
       "Mathpix                                        0.199                    0.278   \n",
       "docling                                        0.801                    0.853   \n",
       "GOT-OCR                                        0.132                    0.204   \n",
       "Nougat                                         0.820                    0.930   \n",
       "GPT4o                                          0.187                    0.281   \n",
       "Qwen2-VL-72B                                   0.148                    0.219   \n",
       "InternVL2-Llama3-76B                           0.184                    0.247   \n",
       "\n",
       "                      data_source: magazine  data_source: academic_literature  \\\n",
       "MinerU-0.9.3                          0.072                             0.025   \n",
       "Marker-0.2.17                         0.134                             0.102   \n",
       "Marker-1.2.3                          0.153                             0.059   \n",
       "Mathpix                               0.138                             0.091   \n",
       "docling                               0.720                             0.326   \n",
       "GOT-OCR                               0.198                             0.179   \n",
       "Nougat                                0.830                             0.214   \n",
       "GPT4o                                 0.173                             0.146   \n",
       "Qwen2-VL-72B                          0.065                             0.315   \n",
       "InternVL2-Llama3-76B                  0.150                             0.419   \n",
       "\n",
       "                      data_source: note  data_source: newspaper   mean  \n",
       "MinerU-0.9.3                      0.984                   0.148  0.189  \n",
       "Marker-0.2.17                     0.470                   0.270  0.255  \n",
       "Marker-1.2.3                      0.651                   0.192  0.259  \n",
       "Mathpix                           0.631                   0.648  0.276  \n",
       "docling                           1.000                   0.996  0.813  \n",
       "GOT-OCR                           0.388                   0.771  0.253  \n",
       "Nougat                            0.991                   0.871  0.817  \n",
       "GPT4o                             0.607                   0.751  0.313  \n",
       "Qwen2-VL-72B                      0.298                   0.790  0.240  \n",
       "InternVL2-Llama3-76B              0.226                   0.903  0.289  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# PDF type\n",
    "dict_list = []\n",
    "\n",
    "for ocr_type in ocr_types:\n",
    "    result_path = os.path.join(result_folder, f'{ocr_type}_{match_name}_metric_result.json')\n",
    "    \n",
    "    with open(result_path, 'r') as f:\n",
    "        result = json.load(f)\n",
    "    \n",
    "    # for category_type in result.keys():\n",
    "    dict_list.append(result['text_block'][\"page\"][\"Edit_dist\"])\n",
    "df2 = pd.DataFrame(dict_list, index=offical_names)\n",
    "\n",
    "reordered_df2 = df2.round(3)\n",
    "\n",
    "selected_columns = reordered_df2[[\"data_source: book\", \"data_source: PPT2PDF\", \"data_source: research_report\", \"data_source: colorful_textbook\", \"data_source: exam_paper\", \"data_source: magazine\", \"data_source: academic_literature\", \"data_source: note\", \"data_source: newspaper\"]]\n",
    "# calculate mean\n",
    "selected_columns['mean'] = selected_columns.mean(axis=1).round(3)\n",
    "# selected_columns['variance'] = selected_columns.var(axis=1)\n",
    "# selected_columns.to_csv('./data_source.csv')\n",
    "\n",
    "selected_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/mnt/petrelfs/ouyanglinke/.local/lib/python3.7/site-packages/ipykernel_launcher.py:19: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/mnt/petrelfs/ouyanglinke/.local/lib/python3.7/site-packages/ipykernel_launcher.py:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fuzzy_scan</th>\n",
       "      <th>watermark</th>\n",
       "      <th>colorful_backgroud</th>\n",
       "      <th>mean</th>\n",
       "      <th>variance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>MinerU-0.9.3</th>\n",
       "      <td>0.150</td>\n",
       "      <td>0.151</td>\n",
       "      <td>0.107</td>\n",
       "      <td>0.136</td>\n",
       "      <td>0.0004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Marker-0.2.17</th>\n",
       "      <td>0.286</td>\n",
       "      <td>0.436</td>\n",
       "      <td>0.290</td>\n",
       "      <td>0.337</td>\n",
       "      <td>0.0049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Marker-1.2.3</th>\n",
       "      <td>0.333</td>\n",
       "      <td>0.484</td>\n",
       "      <td>0.319</td>\n",
       "      <td>0.379</td>\n",
       "      <td>0.0056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mathpix</th>\n",
       "      <td>0.294</td>\n",
       "      <td>0.290</td>\n",
       "      <td>0.182</td>\n",
       "      <td>0.255</td>\n",
       "      <td>0.0027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>docling</th>\n",
       "      <td>0.938</td>\n",
       "      <td>0.807</td>\n",
       "      <td>0.827</td>\n",
       "      <td>0.857</td>\n",
       "      <td>0.0033</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GOT-OCR</th>\n",
       "      <td>0.175</td>\n",
       "      <td>0.190</td>\n",
       "      <td>0.186</td>\n",
       "      <td>0.184</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Nougat</th>\n",
       "      <td>0.934</td>\n",
       "      <td>0.915</td>\n",
       "      <td>0.873</td>\n",
       "      <td>0.907</td>\n",
       "      <td>0.0006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GPT4o</th>\n",
       "      <td>0.263</td>\n",
       "      <td>0.195</td>\n",
       "      <td>0.184</td>\n",
       "      <td>0.214</td>\n",
       "      <td>0.0012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Qwen2-VL-72B</th>\n",
       "      <td>0.101</td>\n",
       "      <td>0.157</td>\n",
       "      <td>0.114</td>\n",
       "      <td>0.124</td>\n",
       "      <td>0.0006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>InternVL2-Llama3-76B</th>\n",
       "      <td>0.120</td>\n",
       "      <td>0.197</td>\n",
       "      <td>0.155</td>\n",
       "      <td>0.157</td>\n",
       "      <td>0.0010</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      fuzzy_scan  watermark  colorful_backgroud   mean  \\\n",
       "MinerU-0.9.3               0.150      0.151               0.107  0.136   \n",
       "Marker-0.2.17              0.286      0.436               0.290  0.337   \n",
       "Marker-1.2.3               0.333      0.484               0.319  0.379   \n",
       "Mathpix                    0.294      0.290               0.182  0.255   \n",
       "docling                    0.938      0.807               0.827  0.857   \n",
       "GOT-OCR                    0.175      0.190               0.186  0.184   \n",
       "Nougat                     0.934      0.915               0.873  0.907   \n",
       "GPT4o                      0.263      0.195               0.184  0.214   \n",
       "Qwen2-VL-72B               0.101      0.157               0.114  0.124   \n",
       "InternVL2-Llama3-76B       0.120      0.197               0.155  0.157   \n",
       "\n",
       "                      variance  \n",
       "MinerU-0.9.3            0.0004  \n",
       "Marker-0.2.17           0.0049  \n",
       "Marker-1.2.3            0.0056  \n",
       "Mathpix                 0.0027  \n",
       "docling                 0.0033  \n",
       "GOT-OCR                 0.0000  \n",
       "Nougat                  0.0006  \n",
       "GPT4o                   0.0012  \n",
       "Qwen2-VL-72B            0.0006  \n",
       "InternVL2-Llama3-76B    0.0010  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# page special issue\n",
    "dict_list = []\n",
    "\n",
    "for ocr_type in ocr_types:\n",
    "    result_path = os.path.join(result_folder, f'{ocr_type}_{match_name}_metric_result.json')\n",
    "    \n",
    "    with open(result_path, 'r') as f:\n",
    "        result = json.load(f)\n",
    "    \n",
    "    # for category_type in result.keys():\n",
    "    dict_list.append(result['text_block'][\"page\"][\"Edit_dist\"])\n",
    "df2 = pd.DataFrame(dict_list, index=offical_names)\n",
    "\n",
    "reordered_df2 = df2.round(3)\n",
    "reordered_df2\n",
    "\n",
    "selected_columns = reordered_df2[['fuzzy_scan', 'watermark', 'colorful_backgroud']]\n",
    "# calculate mean and variance\n",
    "selected_columns['mean'] = selected_columns.mean(axis=1).round(3)\n",
    "selected_columns['variance'] = selected_columns.var(axis=1).round(4)\n",
    "\n",
    "# selected_columns.to_csv('./page_issue.csv')\n",
    "selected_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/mnt/petrelfs/ouyanglinke/.local/lib/python3.7/site-packages/ipykernel_launcher.py:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/mnt/petrelfs/ouyanglinke/.local/lib/python3.7/site-packages/ipykernel_launcher.py:21: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>layout: single_column</th>\n",
       "      <th>layout: double_column</th>\n",
       "      <th>layout: three_column</th>\n",
       "      <th>layout: other_layout</th>\n",
       "      <th>mean</th>\n",
       "      <th>variance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>MinerU-0.9.3</th>\n",
       "      <td>0.311</td>\n",
       "      <td>0.101</td>\n",
       "      <td>0.117</td>\n",
       "      <td>0.376</td>\n",
       "      <td>0.226</td>\n",
       "      <td>0.0143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Marker-0.2.17</th>\n",
       "      <td>0.231</td>\n",
       "      <td>0.251</td>\n",
       "      <td>0.309</td>\n",
       "      <td>0.378</td>\n",
       "      <td>0.292</td>\n",
       "      <td>0.0033</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Marker-1.2.3</th>\n",
       "      <td>0.299</td>\n",
       "      <td>0.299</td>\n",
       "      <td>0.149</td>\n",
       "      <td>0.363</td>\n",
       "      <td>0.277</td>\n",
       "      <td>0.0062</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mathpix</th>\n",
       "      <td>0.189</td>\n",
       "      <td>0.175</td>\n",
       "      <td>0.225</td>\n",
       "      <td>0.413</td>\n",
       "      <td>0.250</td>\n",
       "      <td>0.0091</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>docling</th>\n",
       "      <td>0.758</td>\n",
       "      <td>0.535</td>\n",
       "      <td>0.829</td>\n",
       "      <td>0.701</td>\n",
       "      <td>0.706</td>\n",
       "      <td>0.0118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GOT-OCR</th>\n",
       "      <td>0.163</td>\n",
       "      <td>0.145</td>\n",
       "      <td>0.272</td>\n",
       "      <td>0.468</td>\n",
       "      <td>0.262</td>\n",
       "      <td>0.0165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Nougat</th>\n",
       "      <td>0.852</td>\n",
       "      <td>0.601</td>\n",
       "      <td>0.676</td>\n",
       "      <td>0.873</td>\n",
       "      <td>0.750</td>\n",
       "      <td>0.0133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GPT4o</th>\n",
       "      <td>0.109</td>\n",
       "      <td>0.204</td>\n",
       "      <td>0.254</td>\n",
       "      <td>0.426</td>\n",
       "      <td>0.248</td>\n",
       "      <td>0.0132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Qwen2-VL-72B</th>\n",
       "      <td>0.098</td>\n",
       "      <td>0.248</td>\n",
       "      <td>0.514</td>\n",
       "      <td>0.429</td>\n",
       "      <td>0.322</td>\n",
       "      <td>0.0260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>InternVL2-Llama3-76B</th>\n",
       "      <td>0.082</td>\n",
       "      <td>0.312</td>\n",
       "      <td>0.682</td>\n",
       "      <td>0.444</td>\n",
       "      <td>0.380</td>\n",
       "      <td>0.0472</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      layout: single_column  layout: double_column  \\\n",
       "MinerU-0.9.3                          0.311                  0.101   \n",
       "Marker-0.2.17                         0.231                  0.251   \n",
       "Marker-1.2.3                          0.299                  0.299   \n",
       "Mathpix                               0.189                  0.175   \n",
       "docling                               0.758                  0.535   \n",
       "GOT-OCR                               0.163                  0.145   \n",
       "Nougat                                0.852                  0.601   \n",
       "GPT4o                                 0.109                  0.204   \n",
       "Qwen2-VL-72B                          0.098                  0.248   \n",
       "InternVL2-Llama3-76B                  0.082                  0.312   \n",
       "\n",
       "                      layout: three_column  layout: other_layout   mean  \\\n",
       "MinerU-0.9.3                         0.117                 0.376  0.226   \n",
       "Marker-0.2.17                        0.309                 0.378  0.292   \n",
       "Marker-1.2.3                         0.149                 0.363  0.277   \n",
       "Mathpix                              0.225                 0.413  0.250   \n",
       "docling                              0.829                 0.701  0.706   \n",
       "GOT-OCR                              0.272                 0.468  0.262   \n",
       "Nougat                               0.676                 0.873  0.750   \n",
       "GPT4o                                0.254                 0.426  0.248   \n",
       "Qwen2-VL-72B                         0.514                 0.429  0.322   \n",
       "InternVL2-Llama3-76B                 0.682                 0.444  0.380   \n",
       "\n",
       "                      variance  \n",
       "MinerU-0.9.3            0.0143  \n",
       "Marker-0.2.17           0.0033  \n",
       "Marker-1.2.3            0.0062  \n",
       "Mathpix                 0.0091  \n",
       "docling                 0.0118  \n",
       "GOT-OCR                 0.0165  \n",
       "Nougat                  0.0133  \n",
       "GPT4o                   0.0132  \n",
       "Qwen2-VL-72B            0.0260  \n",
       "InternVL2-Llama3-76B    0.0472  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reading order under different layout\n",
    "dict_list = []\n",
    "\n",
    "for ocr_type in ocr_types:\n",
    "    result_path = os.path.join(result_folder, f'{ocr_type}_{match_name}_metric_result.json')\n",
    "    \n",
    "    with open(result_path, 'r') as f:\n",
    "        result = json.load(f)\n",
    "    \n",
    "    # for category_type in result.keys():\n",
    "    dict_list.append(result['reading_order'][\"page\"][\"Edit_dist\"])\n",
    "\n",
    "df3 = pd.DataFrame(dict_list, index=offical_names)\n",
    "\n",
    "reordered_df3 = df3.round(3)\n",
    "\n",
    "\n",
    "selected_columns3 = reordered_df3[[\"layout: single_column\", \"layout: double_column\", \"layout: three_column\", \"layout: other_layout\"]]\n",
    "# calculate mean and variance\n",
    "selected_columns3['mean'] = selected_columns3.mean(axis=1).round(3)\n",
    "selected_columns3['variance'] = selected_columns3.var(axis=1).round(4)\n",
    "\n",
    "# selected_columns3.to_csv('./layout.csv')\n",
    "selected_columns3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_language: text_english</th>\n",
       "      <th>text_language: text_simplified_chinese</th>\n",
       "      <th>text_language: text_en_ch_mixed</th>\n",
       "      <th>text_background: white</th>\n",
       "      <th>text_background: single_colored</th>\n",
       "      <th>text_background: multi_colored</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>MinerU-0.9.3</th>\n",
       "      <td>0.124</td>\n",
       "      <td>0.206</td>\n",
       "      <td>0.742</td>\n",
       "      <td>0.163</td>\n",
       "      <td>0.147</td>\n",
       "      <td>0.513</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Marker-0.2.17</th>\n",
       "      <td>0.267</td>\n",
       "      <td>0.390</td>\n",
       "      <td>0.499</td>\n",
       "      <td>0.340</td>\n",
       "      <td>0.389</td>\n",
       "      <td>0.497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Marker-1.2.3</th>\n",
       "      <td>0.163</td>\n",
       "      <td>0.379</td>\n",
       "      <td>0.747</td>\n",
       "      <td>0.303</td>\n",
       "      <td>0.396</td>\n",
       "      <td>0.594</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mathpix</th>\n",
       "      <td>0.173</td>\n",
       "      <td>0.774</td>\n",
       "      <td>0.538</td>\n",
       "      <td>0.675</td>\n",
       "      <td>0.554</td>\n",
       "      <td>0.570</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>docling</th>\n",
       "      <td>0.538</td>\n",
       "      <td>0.980</td>\n",
       "      <td>0.931</td>\n",
       "      <td>0.859</td>\n",
       "      <td>0.798</td>\n",
       "      <td>0.927</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GOT-OCR</th>\n",
       "      <td>0.252</td>\n",
       "      <td>0.763</td>\n",
       "      <td>0.266</td>\n",
       "      <td>0.669</td>\n",
       "      <td>0.595</td>\n",
       "      <td>0.440</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Nougat</th>\n",
       "      <td>0.588</td>\n",
       "      <td>0.991</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.875</td>\n",
       "      <td>0.935</td>\n",
       "      <td>0.972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GPT4o</th>\n",
       "      <td>0.171</td>\n",
       "      <td>0.647</td>\n",
       "      <td>0.322</td>\n",
       "      <td>0.536</td>\n",
       "      <td>0.423</td>\n",
       "      <td>0.406</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Qwen2-VL-72B</th>\n",
       "      <td>0.337</td>\n",
       "      <td>0.575</td>\n",
       "      <td>0.310</td>\n",
       "      <td>0.537</td>\n",
       "      <td>0.400</td>\n",
       "      <td>0.233</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>InternVL2-Llama3-76B</th>\n",
       "      <td>0.418</td>\n",
       "      <td>0.606</td>\n",
       "      <td>0.251</td>\n",
       "      <td>0.589</td>\n",
       "      <td>0.366</td>\n",
       "      <td>0.221</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      text_language: text_english  \\\n",
       "MinerU-0.9.3                                0.124   \n",
       "Marker-0.2.17                               0.267   \n",
       "Marker-1.2.3                                0.163   \n",
       "Mathpix                                     0.173   \n",
       "docling                                     0.538   \n",
       "GOT-OCR                                     0.252   \n",
       "Nougat                                      0.588   \n",
       "GPT4o                                       0.171   \n",
       "Qwen2-VL-72B                                0.337   \n",
       "InternVL2-Llama3-76B                        0.418   \n",
       "\n",
       "                      text_language: text_simplified_chinese  \\\n",
       "MinerU-0.9.3                                           0.206   \n",
       "Marker-0.2.17                                          0.390   \n",
       "Marker-1.2.3                                           0.379   \n",
       "Mathpix                                                0.774   \n",
       "docling                                                0.980   \n",
       "GOT-OCR                                                0.763   \n",
       "Nougat                                                 0.991   \n",
       "GPT4o                                                  0.647   \n",
       "Qwen2-VL-72B                                           0.575   \n",
       "InternVL2-Llama3-76B                                   0.606   \n",
       "\n",
       "                      text_language: text_en_ch_mixed  text_background: white  \\\n",
       "MinerU-0.9.3                                    0.742                   0.163   \n",
       "Marker-0.2.17                                   0.499                   0.340   \n",
       "Marker-1.2.3                                    0.747                   0.303   \n",
       "Mathpix                                         0.538                   0.675   \n",
       "docling                                         0.931                   0.859   \n",
       "GOT-OCR                                         0.266                   0.669   \n",
       "Nougat                                          0.983                   0.875   \n",
       "GPT4o                                           0.322                   0.536   \n",
       "Qwen2-VL-72B                                    0.310                   0.537   \n",
       "InternVL2-Llama3-76B                            0.251                   0.589   \n",
       "\n",
       "                      text_background: single_colored  \\\n",
       "MinerU-0.9.3                                    0.147   \n",
       "Marker-0.2.17                                   0.389   \n",
       "Marker-1.2.3                                    0.396   \n",
       "Mathpix                                         0.554   \n",
       "docling                                         0.798   \n",
       "GOT-OCR                                         0.595   \n",
       "Nougat                                          0.935   \n",
       "GPT4o                                           0.423   \n",
       "Qwen2-VL-72B                                    0.400   \n",
       "InternVL2-Llama3-76B                            0.366   \n",
       "\n",
       "                      text_background: multi_colored  \n",
       "MinerU-0.9.3                                   0.513  \n",
       "Marker-0.2.17                                  0.497  \n",
       "Marker-1.2.3                                   0.594  \n",
       "Mathpix                                        0.570  \n",
       "docling                                        0.927  \n",
       "GOT-OCR                                        0.440  \n",
       "Nougat                                         0.972  \n",
       "GPT4o                                          0.406  \n",
       "Qwen2-VL-72B                                   0.233  \n",
       "InternVL2-Llama3-76B                           0.221  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# text attribute\n",
    "dict_list = []\n",
    "\n",
    "for ocr_type in ocr_types:\n",
    "    result_path = os.path.join(result_folder, f'{ocr_type}_{match_name}_metric_result.json')\n",
    "    \n",
    "    with open(result_path, 'r') as f:\n",
    "        result = json.load(f)\n",
    "    \n",
    "    # for category_type in result.keys():\n",
    "    dict_list.append(result['text_block'][\"group\"][\"Edit_dist\"])\n",
    "\n",
    "df4 = pd.DataFrame(dict_list, index=offical_names)\n",
    "df4 = df4.round(3)\n",
    "\n",
    "selected_columns = df4[[\"text_language: text_english\", \"text_language: text_simplified_chinese\", \"text_language: text_en_ch_mixed\", \"text_background: white\", \"text_background: single_colored\", \"text_background: multi_colored\"]]\n",
    "\n",
    "# selected_columns.to_csv('.text_attribute.csv')\n",
    "selected_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>language: table_en</th>\n",
       "      <th>language: table_simplified_chinese</th>\n",
       "      <th>language: table_en_ch_mixed</th>\n",
       "      <th>line: full_line</th>\n",
       "      <th>line: less_line</th>\n",
       "      <th>line: fewer_line</th>\n",
       "      <th>line: wireless_line</th>\n",
       "      <th>with_span: True</th>\n",
       "      <th>with_span: False</th>\n",
       "      <th>include_equation: True</th>\n",
       "      <th>include_equation: False</th>\n",
       "      <th>include_background: True</th>\n",
       "      <th>include_background: False</th>\n",
       "      <th>table_layout: vertical</th>\n",
       "      <th>table_layout: horizontal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>MinerU-0.9.3</th>\n",
       "      <td>75.1</td>\n",
       "      <td>59.3</td>\n",
       "      <td>79.1</td>\n",
       "      <td>59.4</td>\n",
       "      <td>71.6</td>\n",
       "      <td>69.7</td>\n",
       "      <td>60.0</td>\n",
       "      <td>63.6</td>\n",
       "      <td>65.3</td>\n",
       "      <td>66.0</td>\n",
       "      <td>64.4</td>\n",
       "      <td>59.2</td>\n",
       "      <td>67.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>65.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Marker-0.2.17</th>\n",
       "      <td>52.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>44.1</td>\n",
       "      <td>41.0</td>\n",
       "      <td>53.8</td>\n",
       "      <td>46.5</td>\n",
       "      <td>51.4</td>\n",
       "      <td>43.3</td>\n",
       "      <td>46.1</td>\n",
       "      <td>42.0</td>\n",
       "      <td>45.8</td>\n",
       "      <td>43.6</td>\n",
       "      <td>45.8</td>\n",
       "      <td>6.3</td>\n",
       "      <td>45.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Marker-1.2.3</th>\n",
       "      <td>64.9</td>\n",
       "      <td>47.3</td>\n",
       "      <td>49.8</td>\n",
       "      <td>44.5</td>\n",
       "      <td>61.8</td>\n",
       "      <td>59.0</td>\n",
       "      <td>63.6</td>\n",
       "      <td>52.6</td>\n",
       "      <td>52.7</td>\n",
       "      <td>53.2</td>\n",
       "      <td>52.5</td>\n",
       "      <td>48.0</td>\n",
       "      <td>54.9</td>\n",
       "      <td>35.5</td>\n",
       "      <td>52.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mathpix</th>\n",
       "      <td>75.4</td>\n",
       "      <td>63.2</td>\n",
       "      <td>71.3</td>\n",
       "      <td>67.4</td>\n",
       "      <td>77.3</td>\n",
       "      <td>66.3</td>\n",
       "      <td>25.5</td>\n",
       "      <td>70.3</td>\n",
       "      <td>65.4</td>\n",
       "      <td>68.7</td>\n",
       "      <td>66.7</td>\n",
       "      <td>59.7</td>\n",
       "      <td>70.8</td>\n",
       "      <td>19.2</td>\n",
       "      <td>67.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>docling</th>\n",
       "      <td>58.2</td>\n",
       "      <td>24.1</td>\n",
       "      <td>30.1</td>\n",
       "      <td>25.9</td>\n",
       "      <td>33.5</td>\n",
       "      <td>48.0</td>\n",
       "      <td>22.7</td>\n",
       "      <td>34.0</td>\n",
       "      <td>34.7</td>\n",
       "      <td>43.8</td>\n",
       "      <td>32.3</td>\n",
       "      <td>28.8</td>\n",
       "      <td>37.3</td>\n",
       "      <td>4.9</td>\n",
       "      <td>35.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GOT-OCR</th>\n",
       "      <td>51.7</td>\n",
       "      <td>46.2</td>\n",
       "      <td>49.0</td>\n",
       "      <td>45.5</td>\n",
       "      <td>48.3</td>\n",
       "      <td>51.3</td>\n",
       "      <td>46.2</td>\n",
       "      <td>46.0</td>\n",
       "      <td>48.9</td>\n",
       "      <td>45.7</td>\n",
       "      <td>48.4</td>\n",
       "      <td>39.8</td>\n",
       "      <td>51.9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>48.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Nougat</th>\n",
       "      <td>36.2</td>\n",
       "      <td>0.3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.1</td>\n",
       "      <td>3.5</td>\n",
       "      <td>22.1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>8.9</td>\n",
       "      <td>21.0</td>\n",
       "      <td>8.7</td>\n",
       "      <td>2.6</td>\n",
       "      <td>15.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GPT4o</th>\n",
       "      <td>71.1</td>\n",
       "      <td>58.0</td>\n",
       "      <td>57.3</td>\n",
       "      <td>62.5</td>\n",
       "      <td>68.7</td>\n",
       "      <td>61.3</td>\n",
       "      <td>31.2</td>\n",
       "      <td>56.8</td>\n",
       "      <td>64.7</td>\n",
       "      <td>60.8</td>\n",
       "      <td>62.2</td>\n",
       "      <td>61.4</td>\n",
       "      <td>62.2</td>\n",
       "      <td>14.2</td>\n",
       "      <td>62.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Qwen2-VL-72B</th>\n",
       "      <td>57.0</td>\n",
       "      <td>62.5</td>\n",
       "      <td>72.1</td>\n",
       "      <td>70.2</td>\n",
       "      <td>63.5</td>\n",
       "      <td>48.1</td>\n",
       "      <td>57.2</td>\n",
       "      <td>49.3</td>\n",
       "      <td>67.6</td>\n",
       "      <td>47.9</td>\n",
       "      <td>64.3</td>\n",
       "      <td>63.1</td>\n",
       "      <td>60.2</td>\n",
       "      <td>41.4</td>\n",
       "      <td>61.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>InternVL2-Llama3-76B</th>\n",
       "      <td>60.9</td>\n",
       "      <td>58.5</td>\n",
       "      <td>65.4</td>\n",
       "      <td>58.8</td>\n",
       "      <td>65.3</td>\n",
       "      <td>58.3</td>\n",
       "      <td>55.6</td>\n",
       "      <td>49.0</td>\n",
       "      <td>65.1</td>\n",
       "      <td>53.3</td>\n",
       "      <td>60.9</td>\n",
       "      <td>58.8</td>\n",
       "      <td>59.8</td>\n",
       "      <td>6.9</td>\n",
       "      <td>60.3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      language: table_en  language: table_simplified_chinese  \\\n",
       "MinerU-0.9.3                        75.1                                59.3   \n",
       "Marker-0.2.17                       52.0                                42.0   \n",
       "Marker-1.2.3                        64.9                                47.3   \n",
       "Mathpix                             75.4                                63.2   \n",
       "docling                             58.2                                24.1   \n",
       "GOT-OCR                             51.7                                46.2   \n",
       "Nougat                              36.2                                 0.3   \n",
       "GPT4o                               71.1                                58.0   \n",
       "Qwen2-VL-72B                        57.0                                62.5   \n",
       "InternVL2-Llama3-76B                60.9                                58.5   \n",
       "\n",
       "                      language: table_en_ch_mixed  line: full_line  \\\n",
       "MinerU-0.9.3                                 79.1             59.4   \n",
       "Marker-0.2.17                                44.1             41.0   \n",
       "Marker-1.2.3                                 49.8             44.5   \n",
       "Mathpix                                      71.3             67.4   \n",
       "docling                                      30.1             25.9   \n",
       "GOT-OCR                                      49.0             45.5   \n",
       "Nougat                                        0.0              6.1   \n",
       "GPT4o                                        57.3             62.5   \n",
       "Qwen2-VL-72B                                 72.1             70.2   \n",
       "InternVL2-Llama3-76B                         65.4             58.8   \n",
       "\n",
       "                      line: less_line  line: fewer_line  line: wireless_line  \\\n",
       "MinerU-0.9.3                     71.6              69.7                 60.0   \n",
       "Marker-0.2.17                    53.8              46.5                 51.4   \n",
       "Marker-1.2.3                     61.8              59.0                 63.6   \n",
       "Mathpix                          77.3              66.3                 25.5   \n",
       "docling                          33.5              48.0                 22.7   \n",
       "GOT-OCR                          48.3              51.3                 46.2   \n",
       "Nougat                            3.5              22.1                  0.0   \n",
       "GPT4o                            68.7              61.3                 31.2   \n",
       "Qwen2-VL-72B                     63.5              48.1                 57.2   \n",
       "InternVL2-Llama3-76B             65.3              58.3                 55.6   \n",
       "\n",
       "                      with_span: True  with_span: False  \\\n",
       "MinerU-0.9.3                     63.6              65.3   \n",
       "Marker-0.2.17                    43.3              46.1   \n",
       "Marker-1.2.3                     52.6              52.7   \n",
       "Mathpix                          70.3              65.4   \n",
       "docling                          34.0              34.7   \n",
       "GOT-OCR                          46.0              48.9   \n",
       "Nougat                           15.0               8.9   \n",
       "GPT4o                            56.8              64.7   \n",
       "Qwen2-VL-72B                     49.3              67.6   \n",
       "InternVL2-Llama3-76B             49.0              65.1   \n",
       "\n",
       "                      include_equation: True  include_equation: False  \\\n",
       "MinerU-0.9.3                            66.0                     64.4   \n",
       "Marker-0.2.17                           42.0                     45.8   \n",
       "Marker-1.2.3                            53.2                     52.5   \n",
       "Mathpix                                 68.7                     66.7   \n",
       "docling                                 43.8                     32.3   \n",
       "GOT-OCR                                 45.7                     48.4   \n",
       "Nougat                                  21.0                      8.7   \n",
       "GPT4o                                   60.8                     62.2   \n",
       "Qwen2-VL-72B                            47.9                     64.3   \n",
       "InternVL2-Llama3-76B                    53.3                     60.9   \n",
       "\n",
       "                      include_background: True  include_background: False  \\\n",
       "MinerU-0.9.3                              59.2                       67.5   \n",
       "Marker-0.2.17                             43.6                       45.8   \n",
       "Marker-1.2.3                              48.0                       54.9   \n",
       "Mathpix                                   59.7                       70.8   \n",
       "docling                                   28.8                       37.3   \n",
       "GOT-OCR                                   39.8                       51.9   \n",
       "Nougat                                     2.6                       15.2   \n",
       "GPT4o                                     61.4                       62.2   \n",
       "Qwen2-VL-72B                              63.1                       60.2   \n",
       "InternVL2-Llama3-76B                      58.8                       59.8   \n",
       "\n",
       "                      table_layout: vertical  table_layout: horizontal  \n",
       "MinerU-0.9.3                             3.0                      65.8  \n",
       "Marker-0.2.17                            6.3                      45.7  \n",
       "Marker-1.2.3                            35.5                      52.9  \n",
       "Mathpix                                 19.2                      67.9  \n",
       "docling                                  4.9                      35.0  \n",
       "GOT-OCR                                  0.0                      48.7  \n",
       "Nougat                                   0.0                      11.2  \n",
       "GPT4o                                   14.2                      62.7  \n",
       "Qwen2-VL-72B                            41.4                      61.5  \n",
       "InternVL2-Llama3-76B                     6.9                      60.3  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# table attribute\n",
    "dict_list = []\n",
    "\n",
    "for ocr_type in ocr_types:\n",
    "    result_path = os.path.join(result_folder, f'{ocr_type}_{match_name}_metric_result.json')\n",
    "    \n",
    "    with open(result_path, 'r') as f:\n",
    "        result = json.load(f)\n",
    "    \n",
    "    # for category_type in result.keys():\n",
    "    dict_list.append(result['table'][\"group\"][\"TEDS\"])\n",
    "\n",
    "df4 = pd.DataFrame(dict_list, index=offical_names)\n",
    "df4 = df4 * 100\n",
    "df4 = df4.round(1)\n",
    "\n",
    "selected_columns = df4[[\"language: table_en\", \"language: table_simplified_chinese\", \"language: table_en_ch_mixed\", \"line: full_line\", \"line: less_line\", \"line: fewer_line\", \"line: wireless_line\", \n",
    "                        \"with_span: True\", \"with_span: False\", \"include_equation: True\", \"include_equation: False\", \"include_background: True\", \"include_background: False\", \"table_layout: vertical\", \"table_layout: horizontal\"]]\n",
    "\n",
    "# selected_columns.to_csv('./table_attribute.csv')\n",
    "selected_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_language: text_english</th>\n",
       "      <th>text_language: text_simplified_chinese</th>\n",
       "      <th>text_language: text_en_ch_mixed</th>\n",
       "      <th>text_background: white</th>\n",
       "      <th>text_background: single_colored</th>\n",
       "      <th>text_background: multi_colored</th>\n",
       "      <th>text_rotate: normal</th>\n",
       "      <th>text_rotate: rotate90</th>\n",
       "      <th>text_rotate: rotate270</th>\n",
       "      <th>text_rotate: horizontal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>OmniDocBench_easyocr_text_ocr</th>\n",
       "      <td>0.26</td>\n",
       "      <td>0.398</td>\n",
       "      <td>0.445</td>\n",
       "      <td>0.366</td>\n",
       "      <td>0.287</td>\n",
       "      <td>0.388</td>\n",
       "      <td>0.360</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.997</td>\n",
       "      <td>0.926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>OmniDocBench_openocr_text_ocr</th>\n",
       "      <td>0.07</td>\n",
       "      <td>0.068</td>\n",
       "      <td>0.106</td>\n",
       "      <td>0.069</td>\n",
       "      <td>0.058</td>\n",
       "      <td>0.081</td>\n",
       "      <td>0.069</td>\n",
       "      <td>0.038</td>\n",
       "      <td>0.891</td>\n",
       "      <td>0.025</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               text_language: text_english  \\\n",
       "OmniDocBench_easyocr_text_ocr                         0.26   \n",
       "OmniDocBench_openocr_text_ocr                         0.07   \n",
       "\n",
       "                               text_language: text_simplified_chinese  \\\n",
       "OmniDocBench_easyocr_text_ocr                                   0.398   \n",
       "OmniDocBench_openocr_text_ocr                                   0.068   \n",
       "\n",
       "                               text_language: text_en_ch_mixed  \\\n",
       "OmniDocBench_easyocr_text_ocr                            0.445   \n",
       "OmniDocBench_openocr_text_ocr                            0.106   \n",
       "\n",
       "                               text_background: white  \\\n",
       "OmniDocBench_easyocr_text_ocr                   0.366   \n",
       "OmniDocBench_openocr_text_ocr                   0.069   \n",
       "\n",
       "                               text_background: single_colored  \\\n",
       "OmniDocBench_easyocr_text_ocr                            0.287   \n",
       "OmniDocBench_openocr_text_ocr                            0.058   \n",
       "\n",
       "                               text_background: multi_colored  \\\n",
       "OmniDocBench_easyocr_text_ocr                           0.388   \n",
       "OmniDocBench_openocr_text_ocr                           0.081   \n",
       "\n",
       "                               text_rotate: normal  text_rotate: rotate90  \\\n",
       "OmniDocBench_easyocr_text_ocr                0.360                  0.970   \n",
       "OmniDocBench_openocr_text_ocr                0.069                  0.038   \n",
       "\n",
       "                               text_rotate: rotate270  text_rotate: horizontal  \n",
       "OmniDocBench_easyocr_text_ocr                   0.997                    0.926  \n",
       "OmniDocBench_openocr_text_ocr                   0.891                    0.025  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# text OCR\n",
    "\n",
    "ocr_types = ['OmniDocBench_easyocr_text_ocr', 'OmniDocBench_openocr_text_ocr']\n",
    "offical_names = ocr_types\n",
    "\n",
    "result_folder = '/mnt/petrelfs/ouyanglinke/OmniDocBench/result'\n",
    "\n",
    "dict_list = []\n",
    "\n",
    "for ocr_type in ocr_types:\n",
    "    result_path = os.path.join(result_folder, f'{ocr_type}_metric_result.json')\n",
    "    \n",
    "    with open(result_path, 'r') as f:\n",
    "        result = json.load(f)\n",
    "    \n",
    "    # for category_type in result.keys():\n",
    "    dict_list.append(result[\"group\"][\"Edit_dist\"])\n",
    "\n",
    "df4 = pd.DataFrame(dict_list, index=offical_names)\n",
    "df4 = df4.round(3)\n",
    "\n",
    "selected_columns = df4[[\"text_language: text_english\", \"text_language: text_simplified_chinese\", \"text_language: text_en_ch_mixed\", \"text_background: white\", \"text_background: single_colored\", \"text_background: multi_colored\", \"text_rotate: normal\", \"text_rotate: rotate90\", \"text_rotate: rotate270\", \"text_rotate: horizontal\"]]\n",
    "\n",
    "# selected_columns.to_csv('.text_attribute.csv')\n",
    "selected_columns"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "layoutlmv3",
   "language": "python",
   "name": "layoutlmv3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
